import type { Metadata } from "next"; import Link from "next/link"; import { notFound } from "next/navigation"; import { ArrowLeft, ChevronDown, RefreshCw } from "lucide-react"; import { getWorkspace } from "@/lib/session"; import { internalApi, InternalApiError, type CrawlJob, type CrawlPage, type CrawlPagesPage } from "@/lib/api"; import { formatBytes, formatDate, formatMs, formatNumber, timeAgo } from "@/lib/format"; import { PageHeader } from "@/components/ui/page-header"; import { Alert } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { CopyButton } from "@/components/ui/copy-button"; import { EmptyState } from "@/components/ui/empty-state"; import { Stat, StatGrid } from "@/components/ui/stat"; import { Table, TableBody, TableCell, TableEmpty, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { CrawlPageStatusBadge, CrawlStatusBadge } from "@/components/dashboard/crawls/crawl-status-badge"; import { CancelCrawlButton } from "@/components/dashboard/crawls/cancel-crawl-button"; export const dynamic = "force-dynamic"; export const metadata: Metadata = { title: "Crawl" }; const PAGE_SIZE = 100; const PREVIEW_CHARS = 2000; const PAGE_STATUSES = ["success", "blocked", "failed"] as const; type SearchParams = Promise<{ cursor?: string | string[]; status?: string | string[] }>; function first(v: string | string[] | undefined): string | undefined { return Array.isArray(v) ? v[0] : v; } function Row({ label, children, mono }: { label: string; children: React.ReactNode; mono?: boolean }) { return (
{label}
{children}
); } function optionValue(v: unknown): string { if (v === undefined || v === null) return "—"; if (Array.isArray(v)) return v.length ? v.join(", ") : "—"; if (typeof v === "object") return JSON.stringify(v); return String(v); } const OPTION_KEYS = ["max_pages", "max_depth", "format", "same_domain", "allow_subdomains", "respect_robots", "use_sitemap", "concurrency", "delay_ms", "timeout", "main_content", "country", "network", "browser", "browser_fallback", "include_patterns", "exclude_patterns", "webhook_url"] as const; export default async function CrawlDetailPage({ params, searchParams }: { params: Promise<{ id: string }>; searchParams: SearchParams }) { const [ws, { id }, sp] = await Promise.all([getWorkspace(), params, searchParams]); if (!/^crawl_[A-Za-z0-9]{4,64}$/.test(id)) notFound(); const cursor = first(sp.cursor)?.trim() || null; const statusFilter = first(sp.status)?.trim() || null; const status = statusFilter && (PAGE_STATUSES as readonly string[]).includes(statusFilter) ? statusFilter : null; let job: CrawlJob | null = null; let pages: CrawlPagesPage = { data: [], next_cursor: null }; let loadError: string | null = null; try { job = await internalApi.getCrawl(ws.project.id, ws.user.id, id); } catch (e) { if (e instanceof InternalApiError && (e.status === 404 || e.code === "NOT_FOUND")) notFound(); loadError = e instanceof InternalApiError ? e.message : "The Fetcha API service is unreachable."; } if (!job) { return (
Crawls {loadError ?? "Unknown error."}
); } try { pages = await internalApi.crawlPages(ws.project.id, ws.user.id, id, { cursor, limit: PAGE_SIZE, status }); } catch (e) { loadError = e instanceof InternalApiError ? e.message : "Could not load the crawled pages."; } const active = job.status === "queued" || job.status === "running"; const stats = job.stats ?? { discovered: 0, fetched: 0, ok: 0, blocked: 0, failed: 0, bytes: 0 }; const hrefFor = (next: { cursor?: string | null; status?: string | null }) => { const q = new URLSearchParams(); const st = next.status === undefined ? status : next.status; if (st) q.set("status", st); if (next.cursor) q.set("cursor", next.cursor); const s = q.toString(); return `/dashboard/crawls/${job!.id}${s ? `?${s}` : ""}`; }; // Only shown for finished jobs (a live elapsed counter would need a client component). const durationMs = job.started_at && job.completed_at ? new Date(job.completed_at).getTime() - new Date(job.started_at).getTime() : null; return (
Crawls {job.label ?? job.domain ?? job.seed_url} } description={ {job.id} · {job.seed_url} · created {timeAgo(job.created_at)} } actions={ <> {active ? ( ) : null} {active ? : null} } /> {job.error ? ( {job.error.message} ) : null} {active ? ( {job.status === "queued" ? "The job is waiting for a worker slot. " : "Pages are being fetched. "} This page does not update on its own; use Refresh to see progress. ) : null}

Pages

{loadError ? ( {loadError} ) : null} URL Status HTTP Depth Bytes Duration Mode {pages.data.length === 0 ? ( {active ? "No pages fetched yet." : status ? `No ${status} pages.` : "No pages were fetched."} ) : ( pages.data.map((p) => ) )}
{formatNumber(pages.data.length)} page{pages.data.length === 1 ? "" : "s"} shown{cursor ? " (continued)" : ""}
{cursor ? ( ) : null} {pages.next_cursor ? ( ) : null}
{!pages.data.length && !active && !loadError && !status ? : null}
); } function FilterLink({ href, active, children }: { href: string; active: boolean; children: React.ReactNode }) { return ( {children} ); } function PageRow({ page: p }: { page: CrawlPage }) { const preview = typeof p.content === "string" && p.content.length ? p.content.slice(0, PREVIEW_CHARS) : null; const truncated = typeof p.content === "string" && p.content.length > PREVIEW_CHARS; return (
{p.url} {p.title ? ( {p.title} ) : null} {p.error_code ? {p.error_code} : null} {p.http_status ?? —} {p.depth} {formatBytes(p.bytes)} {formatMs(p.duration_ms)} {p.mode ? {p.mode === "browser" ? "Browser" : "HTTP"} : —}
{p.final_url && p.final_url !== p.url ? ( <>
Final URL
{p.final_url}
) : null} {p.description ? ( <>
Description
{p.description}
) : null}
Content type
{p.content_type ?? "—"}
Links
{p.links_count ?? "—"}
Fetched
{p.fetched_at ? formatDate(p.fetched_at, { timeStyle: "medium" }) : "—"}
{preview ? ( <>
{preview}
{truncated ?

Showing the first {formatNumber(PREVIEW_CHARS)} of {formatNumber(p.content!.length)} characters.

: null} ) : (

No stored content for this page.

)}
); }